
package co.edu.unal.bioingenium.kbmed.knowledge.mapping.main;

import co.edu.unal.bioingenium.kbmed.config.Configuration;
import co.edu.unal.bioingenium.kbmed.knowledge.ontology.OntologyMetaData;
import co.edu.unal.bioingenium.kbmed.knowledge.ontology.dao.KnowledgeSourceDAO;
import co.edu.unal.bioingenium.kbmed.knowledge.ontology.dao.KnowledgeSourceDAOFactory;
import co.edu.unal.bioingenium.kbmed.knowledge.snomed.SnomedMetaData;
import co.edu.unal.bioingenium.kbmed.text.filter.FilterPipe;
import co.edu.unal.bioingenium.kbmed.text.index.structures.InvertedIndexStrToStr;
import co.edu.unal.bioingenium.kbmed.text.index.structures.InvertedIndexStrToWordInContext;
import co.edu.unal.bioingenium.kbmed.text.index.inverted.io.InvertedIndexIO;
import co.edu.unal.bioingenium.kbmed.text.representation.WordList;
import co.edu.unal.bioingenium.kbmed.text.representation.io.WordListIO;
import ds.tree.RadixTreeImpl;
import ds.tree.io.PrefixIndexIO;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 *
 * @author lariverosc
 */
public class IndexOntologyMain {

    /**
     * @param:
     * @return:
     */
    public static void buildIndexByConcepts(OntologyMetaData ontologyMetaData) {
        try {
            //Creating and storing the the word by concepts posting list's
            InvertedIndexStrToStr wordByConceptIndex = new InvertedIndexStrToStr();
            WordList wordList = new WordList(Configuration.KNOWLEDGE_SOURCE_NAME);
            FilterPipe pipeline = new FilterPipe();
            pipeline.setMinTokenLength(3);
            pipeline.setRemoveNumbers(true);
            KnowledgeSourceDAO knowledgeSourceDAO = KnowledgeSourceDAOFactory.getKnowledgeSourceDAO(ontologyMetaData);
            System.out.println("Indexing by concepts " + ontologyMetaData.getName());
            Set<String> conceptsIds = knowledgeSourceDAO.getConceptsIds(ontologyMetaData.getConceptStatusActive());
            int totalConcepts = conceptsIds.size();
            Map<String, String> conceptDescriptions;
            List<String> words;
            int i = 1;
            for (String conceptId : conceptsIds) {//For each concept
                conceptDescriptions = knowledgeSourceDAO.getTermsForConcept(conceptId, ontologyMetaData.getDescriptorStatusActive());
                for (String descriptionId : conceptDescriptions.keySet()) {//For each descriptor associated with the concept
                    words = pipeline.filter(conceptDescriptions.get(descriptionId));
                    for (String word : words) {//For each word in the descriptor
                        //To index
                        wordByConceptIndex.addElement(word, conceptId); //Add word-concept pair to index
                        //To wordlist
                        wordList.addWord(word); //Add word to WordList
                        //To counts
                        wordList.addGlobalOcurrence(word);// Add word global ocurrence
                        if (!wordByConceptIndex.getElement(word).contains(conceptId)) {
                            wordList.addInObjectOccurence(word);// Add in object occurrence
                        }
                    }
                }
                System.out.println((++i) + "/" + totalConcepts + " indexing " + conceptId);
            }
            wordByConceptIndex.pruneAbove(2000);
            System.out.println(wordList.getNumWords() + " words in " + ontologyMetaData.getName());
            InvertedIndexIO.storeStrToStrInvertedIndex(wordByConceptIndex, Configuration.KNOWLEDGE_SOURCE_NAME + "WordByConcept.idx");
            WordListIO.storeWordList(wordList, Configuration.KNOWLEDGE_SOURCE_NAME + "WordByConcept.lst");
            buildPrefixIndex(wordList.getSortedWordList(), Configuration.KNOWLEDGE_SOURCE_NAME + "ConceptsWordsPrefix.idx");
        } catch (IOException ex) {
            Logger.getLogger(IndexOntologyMain.class.getName()).log(Level.SEVERE, null, ex);
        }
    }

    /**
     * @param:
     * @return:
     */
    public static void buildIndexByDescriptors(OntologyMetaData ontologyMetaData) {
        try {
            //Creating and storing the the word by concepts posting list's
            InvertedIndexStrToStr wordByDescriptionIndex = new InvertedIndexStrToStr();
            WordList wordList = new WordList(Configuration.KNOWLEDGE_SOURCE_NAME);
            FilterPipe pipeline = new FilterPipe();
            pipeline.setMinTokenLength(3);
            pipeline.setRemoveNumbers(true);
            KnowledgeSourceDAO knowledgeSourceDAO = KnowledgeSourceDAOFactory.getKnowledgeSourceDAO(ontologyMetaData);
            System.out.println("Indexing by descriptors " + ontologyMetaData.getName());
            Set<String> conceptsIds = knowledgeSourceDAO.getConceptsIds(ontologyMetaData.getConceptStatusActive());
            System.out.println(conceptsIds.size() + " concepts in " + ontologyMetaData.getName());
            int totalConcepts = conceptsIds.size();
            Map<String, String> descriptions;
            List<String> words;
            int i = 1;
            for (String conceptId : conceptsIds) {//For each concept
                descriptions = knowledgeSourceDAO.getTermsForConcept(conceptId, ontologyMetaData.getDescriptorStatusActive());
                for (String descriptionId : descriptions.keySet()) { //For each descriptor associated with the concept
                    words = pipeline.filter(descriptions.get(descriptionId));
                    for (String word : words) {//For each word in the descriptor
                        //To index
                        wordByDescriptionIndex.addElement(word, descriptionId); //Add word-descriptor pair to index
                        //To wordlist
                        wordList.addWord(word); //Add word to WordList
                        //To counts
                        wordList.addGlobalOcurrence(word);// Add word global ocurrence
                        if (!wordByDescriptionIndex.getElement(word).contains(descriptionId)) {
                            wordList.addInObjectOccurence(word); // Add in object occurrence
                        }
                    }
                }
                System.out.println((++i) + "/" + totalConcepts + " indexing " + conceptId);
            }
            wordByDescriptionIndex.pruneAbove(2000);
            System.out.println(wordList.getNumWords() + " words in " + ontologyMetaData.getName());
            InvertedIndexIO.storeStrToStrInvertedIndex(wordByDescriptionIndex, Configuration.KNOWLEDGE_SOURCE_NAME + "WordByDescriptor.idx");
            WordListIO.storeWordList(wordList, Configuration.KNOWLEDGE_SOURCE_NAME + "WordByDescriptor.lst");
            buildPrefixIndex(wordList.getSortedWordList(), Configuration.KNOWLEDGE_SOURCE_NAME + "DescriptorsWordsPrefix.idx");
        } catch (IOException ex) {
            Logger.getLogger(IndexOntologyMain.class.getName()).log(Level.SEVERE, null, ex);
        }
    }

    /**
     * @param:
     * @return:
     */
    public static void buildIndexByConceptsWithContext(OntologyMetaData ontologyMetaData) {
        try {
            //Creating and storing the the word by concepts posting list's
            InvertedIndexStrToWordInContext wordByConceptContextIndex = new InvertedIndexStrToWordInContext();
            WordList wordList = new WordList(Configuration.KNOWLEDGE_SOURCE_NAME);
            FilterPipe pipeline = new FilterPipe();
            pipeline.setMinTokenLength(3);
            pipeline.setRemoveNumbers(true);
            KnowledgeSourceDAO knowledgeSourceDAO = KnowledgeSourceDAOFactory.getKnowledgeSourceDAO(ontologyMetaData);
            System.out.println("Indexing by concepts " + ontologyMetaData.getName());
            Set<String> conceptsIds = knowledgeSourceDAO.getConceptsIds(ontologyMetaData.getConceptStatusActive());
            int totalConcepts = conceptsIds.size();
            Map<String, String> conceptDescriptions;
            List<String> words;
            int i = 1;
            short position;
            for (String conceptId : conceptsIds) {//For each concept
                conceptDescriptions = knowledgeSourceDAO.getTermsForConcept(conceptId, ontologyMetaData.getDescriptorStatusActive());
                for (String descriptionId : conceptDescriptions.keySet()) {//For each descriptor associated with the concept
                    words = pipeline.filter(conceptDescriptions.get(descriptionId));
                    position = 0;
                    for (String word : words) {//For each word in the descriptor
                        //To index
                        wordByConceptContextIndex.addElement(word, conceptId, position); //Add word-concept pair to index
                        //To wordlist
                        wordList.addWord(word); //Add word to WordList
                        //To counts
                        wordList.addGlobalOcurrence(word);// Add word global ocurrence
                        if (!wordByConceptContextIndex.getElement(word).containsKey(conceptId)) {
                            wordList.addInObjectOccurence(word);// Add in object occurrence
                        }
                        position++;
                    }
                }
                System.out.println((++i) + "/" + totalConcepts + " indexing " + conceptId);
            }
            wordByConceptContextIndex.pruneAbove(2000);
            System.out.println(wordList.getNumWords() + " words in " + ontologyMetaData.getName());
            InvertedIndexIO.storeStrToWordInContextInvertedIndex(wordByConceptContextIndex, Configuration.KNOWLEDGE_SOURCE_NAME + "WordByConceptCtx.idx");
            WordListIO.storeWordList(wordList, Configuration.KNOWLEDGE_SOURCE_NAME + "WordByConcept.lst");
            buildPrefixIndex(wordList.getSortedWordList(), Configuration.KNOWLEDGE_SOURCE_NAME + "ConceptsWordsPrefix.idx");
        } catch (IOException ex) {
            Logger.getLogger(IndexOntologyMain.class.getName()).log(Level.SEVERE, null, ex);
        }
    }

    /**
     * @param:
     * @return:
     */
    public static void buildIndexByDescriptorsWithContext(OntologyMetaData ontologyMetaData) {
        try {
            //Creating and storing the the word by concepts posting list's
            InvertedIndexStrToWordInContext wordByDescriptionContextIndex = new InvertedIndexStrToWordInContext();
            WordList wordList = new WordList(Configuration.KNOWLEDGE_SOURCE_NAME);
            FilterPipe pipeline = new FilterPipe();
            pipeline.setMinTokenLength(3);
            pipeline.setRemoveNumbers(true);
            KnowledgeSourceDAO knowledgeSourceDAO = KnowledgeSourceDAOFactory.getKnowledgeSourceDAO(ontologyMetaData);
            System.out.println("Indexing by descriptors " + ontologyMetaData.getName());
            Set<String> conceptsIds = knowledgeSourceDAO.getConceptsIds(ontologyMetaData.getConceptStatusActive());
            System.out.println(conceptsIds.size() + " concepts in " + ontologyMetaData.getName());
            int totalConcepts = conceptsIds.size();
            Map<String, String> descriptions;
            List<String> words;
            int i = 1;
            short position;
            for (String conceptId : conceptsIds) {//For each concept
                descriptions = knowledgeSourceDAO.getTermsForConcept(conceptId, ontologyMetaData.getDescriptorStatusActive());
                for (String descriptionId : descriptions.keySet()) { //For each descriptor associated with the concept
                    words = pipeline.filter(descriptions.get(descriptionId));
                    position = 0;
                    for (String word : words) {//For each word in the descriptor
                        //To index
                        wordByDescriptionContextIndex.addElement(word, descriptionId, position); //Add word-descriptor pair to index
                        //To wordlist
                        wordList.addWord(word); //Add word to WordList
                        //To counts
                        wordList.addGlobalOcurrence(word);// Add word global ocurrence
                        if (!wordByDescriptionContextIndex.getElement(word).containsKey(descriptionId)) {
                            wordList.addInObjectOccurence(word); // Add in object occurrence
                        }
                        position++;
                    }
                }
                System.out.println((++i) + "/" + totalConcepts + " indexing " + conceptId);
            }
            wordByDescriptionContextIndex.pruneAbove(2000);
            System.out.println(wordList.getNumWords() + " words in " + ontologyMetaData.getName());
            InvertedIndexIO.storeStrToWordInContextInvertedIndex(wordByDescriptionContextIndex, Configuration.KNOWLEDGE_SOURCE_NAME + "WordByDescriptorCtx.idx");
            WordListIO.storeWordList(wordList, Configuration.KNOWLEDGE_SOURCE_NAME + "WordByDescriptor.lst");
            buildPrefixIndex(wordList.getSortedWordList(), Configuration.KNOWLEDGE_SOURCE_NAME + "DescriptorsWordsPrefix.idx");
        } catch (IOException ex) {
            Logger.getLogger(IndexOntologyMain.class.getName()).log(Level.SEVERE, null, ex);
        }
    }

    
    
    
    /**
     * @param:
     * @return:
     */
    public static void buildPrefixIndex(List<String> words, String name) {
        try {
            System.out.println("Creating the prefix tree for all words.");
            RadixTreeImpl<Integer> prefixIndex = new RadixTreeImpl<Integer>();
            Integer count = 0;
            String subString;
            for (String word : words) {
                //TODO this index could be better indexing entire words and preserving intermediate values
//                for (int i = 0; i < word.length(); i++) {
//                    subString = word.substring(0, i);
//                    if (!prefixIndex.contains(subString)) {
//                        prefixIndex.insert(subString, count);
//                    }
//                }
                if (word.length() >= 4) {
                    subString = word.substring(0, 4);
                    if (!prefixIndex.contains(subString)) {
                        prefixIndex.insert(subString, count);
                    }
                } else {
                    if (!prefixIndex.contains(word)) {
                        prefixIndex.insert(word, count);
                    }
                }
                count++;
            }
            PrefixIndexIO.store(prefixIndex, name);
        } catch (IOException ex) {
            Logger.getLogger(IndexOntologyMain.class.getName()).log(Level.SEVERE, "Error while save the prefix index", ex);
        }

    }

    /**
     * @param:
     * @return:
     */
    public static void main(String... args) {
        OntologyMetaData ontologyMetaData = new SnomedMetaData();
//        buildIndexByDescriptors(ontologyMetaData);
//        buildIndexByConcepts(ontologyMetaData);
        buildIndexByDescriptorsWithContext(ontologyMetaData);
        buildIndexByConceptsWithContext(ontologyMetaData);
        
    }
}
